# -*- coding: utf-8 -*-
import Queue,sys
import threading
import urllib2
import time
from BeautifulSoup import BeautifulSoup
from mysql_class import MYSQL

queue = Queue.Queue()
out_queue = Queue.Queue()

class ThreadUrl(threading.Thread):
    def __init__(self, queue, out_queue):
        threading.Thread.__init__(self)
        self.queue = queue
        self.out_queue = out_queue

    
    def run(self):
        while True:
            try:
                host = self.queue.get()		
                #执行我们要执行的访法
                time.sleep(0.2)
                url = urllib2.urlopen(host)
                chunk = url.read()

                #place chunk into out queue 把回去的结果放进任务队列中
                self.out_queue.put(chunk)

                self.queue.task_done()
            except:
                print sys.exc_info()
                raise
		    

class DatamineThread(threading.Thread):
    def __init__(self, out_queue):
        threading.Thread.__init__(self)
        self.out_queue = out_queue

    def run(self):
        try:
            chunk = self.out_queue.get()

            soup = BeautifulSoup(chunk)
            stritem = ''
            items =  soup.findAll(attrs={'class' : 'domainurl'})
            openfile  = open('domain.txt', 'w')
            for item in items:
               
                sql = "INSERT INTO content, addtime VALUES ('%s', %d)" % (item, time.time())
               
                #print sql
                #MYSQL(sql)
                item = str(item)
                stritem += item+"\n"
            
            openfile.write(item)
        
            self.out_queue.task_done()
        
            openfile.close()
        except:
            print sys.exc_info()
            raise
           
start = time.time()

def main():
    links = ['http://del.chinaz.com/?pagesize=30&date=1&page=%d' %i for i in range(1,3)]
    totals = len(links)
    print totals
    #
    for i in range(totals):
        t = ThreadUrl(queue, out_queue)
        t.setDaemon(True)
        t.start()

    for host in links:
        queue.put(host)

    for i in range(totals):
        dt = DatamineThread(out_queue)
        dt.setDaemon(True)
        dt.start()

    queue.join()
    out_queue.join()

main()

print "Elapsed Time: %s " % (time.time() - start)
